package com.translation.gutenberg;

import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.translation.ArrayInterator;
import com.translation.FileReader;
import com.translation.dao.BaseDAO;
import com.translation.dao.Batch;

public class GutenbergImporter {

	public static final Pattern StartPattern = Pattern
			.compile("\\*\\*\\*[\\s]*START OF THIS PROJECT GUTENBERG EBOOK.*?\\*\\*\\*");
	public static final Pattern EndPattern = Pattern
			.compile("\\*\\*\\*[\\s]*END OF THIS PROJECT GUTENBERG EBOOK.*?\\*\\*\\*");
	public static final Pattern TokenPattern = Pattern.compile("([^\\p{Punct}\\s\\d¡]([\\-\\']|[^\\p{Punct}\\s¡])*)");
	protected ThreadPoolExecutor pool = new ThreadPoolExecutor(200, 1000, 10, TimeUnit.HOURS,
			new ArrayBlockingQueue<Runnable>(1000, false), new ThreadPoolExecutor.CallerRunsPolicy());
	protected BaseDAO dao = new BaseDAO();
	protected FilenameFilter filter = new FilenameFilter() {
		@Override
		public boolean accept(File dir, String name) {
			return name.endsWith(".txt");
		}
	};

	protected Queue<String> tokens = new LinkedBlockingQueue<String>(1000000);
	protected Catalog catalog;
	protected Map<String, Boolean> list = new ConcurrentHashMap<String, Boolean>(100000);

	public GutenbergImporter() {
		init();
	}

	private void init() {
		catalog = new Catalog();
	}

	public class ImporterThread extends Thread {
		public void run() {
			while (list.size() > 0) {
				try {
					Thread.sleep(1000);
				} catch (Exception e) {
				}
			}
			batch();
		}
	}

	public class FileImporter implements Runnable {

		private File file;

		public FileImporter(File f) {
			this.file = f;
		}

		@Override
		public void run() {
			try {
				System.out.println("Start ImportFile:" + file.getAbsolutePath());
				importFile(file);
				System.out.println("End: ImportFile:" + file.getAbsolutePath());
			} catch (Exception e) {
				System.out.println("Error:" + file.getName() + "," + e.getMessage());
			} finally {
				list.remove(file.getAbsolutePath());
			}
		}

	}

	public static void main(String[] args) {
		GutenbergImporter i = new GutenbergImporter();
		System.out.println("Start:" + new Date());
		i.importDirs(args);
		System.out.println("End:" + new Date());
	}

	public void importDirs(String... args) {
		for (String fname : args) {
			File dir = new File(fname);
			this.listDir(dir);
		}

		this.importer();

		for (String fname : args) {
			File dir = new File(fname);
			this.importDir(dir);
		}
	}

	private void importer() {
		Thread th = new ImporterThread();
		pool.execute(th);
	}

	public void batch() {
		try {
			Set<String> s = new HashSet<String>(1000000);

			String[] arr = new String[tokens.size()];
			tokens.toArray(arr);

			for (int i = 0; i < arr.length; i++) {
				s.add(arr[i]);
			}
			int size = s.size();
			final String[] values = new String[s.size()];
			s.toArray(values);

			Iterator<String> it = new ArrayInterator<String>(values);

			/*
			Iterator<String> it = tokens.iterator();
			String str = "";
			while ((str = it.next()) != null) {
				System.out.println(str.toLowerCase());
			}*/

			Batch batch = getBatch(it);
			dao.namedBatch("index", batch);
			System.out.println("Batch:" + new Date() + ", size:" + size);
		} catch (Exception e) {
			System.out.println("#Exception:" + e.getMessage());
		} finally {
			shutdown();
		}
	}

	protected void shutdown() {
		pool.shutdown();
	}

	public void listDir(File dir) {
		File[] files = dir.listFiles(filter);

		for (File f : files) {
			list.put(f.getAbsolutePath(), true);
		}
	}

	public void importDir(File dir) {
		File[] files = dir.listFiles(filter);

		for (File f : files) {
			FileImporter r = new FileImporter(f);
			pool.execute(r);
		}
	}

	public void importFile(File f) throws IOException, SQLException {
		FileReader r = new FileReader();
		String txt = r.read(f);
		txt = contents(txt);
		importContents(txt);
	}

	private void importContents(final String txt) throws SQLException, IOException {
		String word = null;
		try {
			Matcher m = TokenPattern.matcher(txt);
			while (m.find()) {
				word = m.group(1).trim().toLowerCase();
				if (word.length() > 255) {
					continue;
				}
				updateFrequency(word);
				importWord(word);
			}
		} catch (Exception e) {
		}
	}

	protected void importWord(String word) {
		if (!catalog.containsKey(word)) {
			tokens.add(word);
		}
	}

	private void updateFrequency(String word) {
		Integer freq = catalog.get(word);
		if (freq == null) {
			freq = 0;
		}
		catalog.put(word, ++freq);
	};

	private Batch getBatch(final Iterator<String> it) {

		return new Batch() {

			private String word;

			@Override
			public void setParams(PreparedStatement st) throws SQLException {
				int index = 1;
				st.setString(index++, word.toLowerCase());
			}

			@Override
			public boolean result(int[] result, PreparedStatement st) throws SQLException {
				return true;
			}

			@Override
			public boolean next() throws SQLException {
				word = it.next();
				return word != null;
			}

			@Override
			public int getSize() {
				return 100;
			}

			@Override
			public void error(SQLException e) {
				if (word != null) {
					System.out.println("Near:" + word);
				}
			}
		};
	}

	private String contents(String txt) {
		Matcher m1 = StartPattern.matcher(txt);
		Matcher m2 = EndPattern.matcher(txt);
		int startIndex = m1.find() ? m1.end() : 0;
		int endIndex = m2.find() ? m2.start() : txt.length();
		return txt.substring(startIndex, endIndex);
	}
}
